{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#这个程序的作用是生成第三层的聚类，并且导出聚类中心\n",
    "#这个程序不假定数据类型，可以是Uint8,int8,Float或者各种类型的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#头文件在这里集中import\n",
    "import numpy as np\n",
    "import time\n",
    "import os\n",
    "import struct\n",
    "import faiss\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#配置参数\n",
    "written = True #是否写入文件。避免测试时覆盖已写入文件\n",
    "evaluation = False #是否测试聚类精度。设置为False则是只训练聚类并导出\n",
    "type = np.float32 #指定数据集和查询集的数据类型。GT集恒为np.uint32\n",
    "# type = np.uint8\n",
    "\n",
    "#设置文件路径.注意所有向量数据集都转化为bin格式的，即头4个字节表示向量数量、紧接着4个表示每个向量的维数，然后是每个向量。里面不包含每个特征的大小，通过程序参数指定\n",
    "#sift1M\n",
    "dataset_path = \"/home/gary/Code/DiskANN/build/data/sift/sift_base.fbin\"\n",
    "query_path = \"/home/gary/Code/DiskANN/build/data/sift/sift_query.fbin\" #如果evaluation设置为False，则这两个参数可以不设置\n",
    "gt_path = \"/home/gary/Code/DiskANN/build/data/sift/sift_groundtruth.fbin\"\n",
    "# #bigann100M\n",
    "# dataset_path = \"/home/gary/Code/DiskANN/build/data/bigann/bigann_learn.bbin\"\n",
    "# query_path = \"/home/gary/Code/DiskANN/build/data/bigann/bigann_query.bbin\"\n",
    "# gt_path = \"/home/gary/Code/DiskANN/build/data/bigann/bigann_gt.ibin\"\n",
    "\n",
    "cluster_center_path = \"/home/gary/Code/vector-ssd/hybrid-index-host/cluster_centers.bin\"\n",
    "\n",
    "#聚类参数\n",
    "cluster_count = 1000 #设置聚类数量\n",
    "train_ratio = 1 #设置训练比例，例如1是全部向量用来训练\n",
    "\n",
    "#查询参数\n",
    "k = 10 #返回多少个近邻\n",
    "nprobe = 10 #查询多少个聚类\n",
    "\n",
    "#DiskANN建图时的参数\n",
    "diskann_path = \"/home/gary/Code/DiskANN/build/apps/build_disk_index\" #diskann程序的路径\n",
    "R = 32  #平均邻居数\n",
    "L = 50  #构建时的候选列表长度\n",
    "B = 120 #目标内存大小(GB)\n",
    "M = 120 #构建时的可用内存，越大越好(GB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取bin格式的数据集文件，数据集、查询集、GT集都可以使用这个函数读取\n",
    "def read_bin(file_path, type):\n",
    "    file_size = os.path.getsize(file_path)\n",
    "    print(\"Read data from\", file_path, \"\\nfile size:\",file_size)\n",
    "    feature_size = np.dtype(type).itemsize\n",
    "    with open(file_path,\"rb\") as fd:\n",
    "        lines = int.from_bytes(fd.read(4), byteorder='little')\n",
    "        dim = int.from_bytes(fd.read(4), byteorder='little')\n",
    "        print(\"lines:\",lines,\"dim:\",dim)\n",
    "\n",
    "        data_size = lines * dim * feature_size\n",
    "        if(data_size+8 != file_size):\n",
    "            print(\"Error! file size and argument not match!\")\n",
    "            return None\n",
    "\n",
    "        binary_data = fd.read(data_size)\n",
    "        vectors = np.frombuffer(binary_data, dtype=type)\n",
    "        vectors = vectors.reshape(lines, dim)\n",
    "        print(\"Returned vector list:\",vectors.shape, vectors.dtype)\n",
    "\n",
    "        return vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read data from /home/gary/Code/DiskANN/build/data/sift/sift_base.fbin \n",
      "file size: 512000008\n",
      "lines: 1000000 dim: 128\n",
      "Returned vector list: (1000000, 128) float32\n",
      "[[  0.  16.  35. ...  25.  23.   1.]\n",
      " [ 14.  35.  19. ...  11.  21.  33.]\n",
      " [  0.   1.   5. ...   4.  23.  10.]\n",
      " ...\n",
      " [ 30.  12.  12. ...  50.  10.   0.]\n",
      " [  0.   5.  12. ...   1.   2.  13.]\n",
      " [114.  31.   0. ...  25.  16.   0.]]\n"
     ]
    }
   ],
   "source": [
    "dataset = read_bin(dataset_path, type)\n",
    "print(dataset)\n",
    "\n",
    "queries = None\n",
    "gts = None\n",
    "if(evaluation):\n",
    "    queries = read_bin(query_path, type)\n",
    "    print(queries)\n",
    "\n",
    "    gts = read_bin(gt_path, np.uint32)\n",
    "    print(gts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start train\n",
      "Train done, spend: 22.2 second\n",
      "Start add vector\n",
      "Add done, spend: 8.4 second\n"
     ]
    }
   ],
   "source": [
    "#用faiss的IVF库来构建IVF索引\n",
    "IVF_index = faiss.index_factory(dataset.shape[1], f\"IVF%d,Flat\"%cluster_count)  # 创建一个IVF索引\n",
    "\n",
    "# 训练索引\n",
    "print(\"Start train\")\n",
    "st = time.time()\n",
    "step = int(1/train_ratio) #步长，即我们间隔多远取一个向量来训练\n",
    "IVF_index.train(dataset[::step])\n",
    "et = time.time()\n",
    "print(\"Train done, spend: %.1f second\"%(et-st))\n",
    "\n",
    "# 添加向量到索引\n",
    "print(\"Start add vector\")\n",
    "st = time.time()\n",
    "IVF_index.add(dataset)\n",
    "et = time.time()\n",
    "print(\"Add done, spend: %.1f second\"%(et-st))\n",
    "\n",
    "#保存索引\n",
    "#创建用户名\n",
    "dataset_name = dataset_path.split(\"/\")[-1].split(\".\")[0]\n",
    "if(written):\n",
    "    faiss.write_index(IVF_index, \"ivf_index_%s_%d.faiss\"%(dataset_name,cluster_count)) #输出的这个作用不大可以不管"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_recall(I, gts, k):\n",
    "    num_queries = I.shape[0]\n",
    "    recall_sum = 0\n",
    "    \n",
    "    for i in range(num_queries):\n",
    "        retrieved = set(I[i])\n",
    "        ground_truth = set(gts[i][:k])  # 取前k个真实结果\n",
    "        correct = len(retrieved.intersection(ground_truth))\n",
    "        recall_sum += correct / k\n",
    "    \n",
    "    return recall_sum / num_queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#聚合测试\n",
    "if(evaluation):\n",
    "    for i in [1,2,4,8,16,32]:\n",
    "        IVF_index.nprobe = i\n",
    "        D, I = IVF_index.search(queries, k)  # 实际上搜索索引\n",
    "        print(compute_recall(I, gts, k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取量化器\n",
    "quantizer = IVF_index.quantizer\n",
    "centroids = np.zeros((cluster_count, dataset.shape[1]), dtype=np.float32)\n",
    "a = IVF_index.quantizer.reconstruct_n(0, cluster_count, centroids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 17.512346     8.728395     8.567902   ...  11.845679     9.030865\n",
      "    7.54321   ]\n",
      " [ 39.294117     6.141711     0.81016046 ...  20.727274    18.355616\n",
      "   15.045455  ]\n",
      " [  9.577465     2.826291     3.8732395  ...   1.0938967    0.9530517\n",
      "    7.1314554 ]\n",
      " ...\n",
      " [ 20.149702     6.8622756    4.3413177  ...   2.9161677    1.5269462\n",
      "    6.2754493 ]\n",
      " [ 12.572992    18.518248    17.489052   ...  15.138686    10.843065\n",
      "   21.167883  ]\n",
      " [102.7822      16.501894     1.0227273  ...  15.606061    14.630682\n",
      "    6.5587125 ]]\n",
      "(1000, 128)\n",
      "float32\n"
     ]
    }
   ],
   "source": [
    "#已验证，此时centroids存储的即聚类的质心\n",
    "print(centroids)\n",
    "print(centroids.shape)\n",
    "print(centroids.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将质心以.bin的格式写入文件中\n",
    "def write_array_to_binary_file(array, filename):\n",
    "    # 检查输入是否为numpy数组\n",
    "    if not isinstance(array, np.ndarray):\n",
    "        raise ValueError(\"Input must be a NumPy array.\")\n",
    "    \n",
    "    # 获取数组的形状（行数和列数）\n",
    "    rows, cols = array.shape\n",
    "    \n",
    "    # 打开文件准备写入\n",
    "    with open(filename, 'wb') as f:\n",
    "        # 将行数和列数写入文件的前8个字节\n",
    "        # 使用>表明大端字节序（Big Endian），I表示无符号整型\n",
    "        np.array(rows, dtype=np.uint32).tofile(f)\n",
    "        np.array(cols, dtype=np.uint32).tofile(f)\n",
    "        \n",
    "        # 写入数组数据\n",
    "        # 确保数据类型正确，这里假设使用float32\n",
    "        array.tofile(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cluster center has written to: /home/gary/Code/vector-ssd/hybrid-index-host/cluster_centers.bin\n"
     ]
    }
   ],
   "source": [
    "if(written):\n",
    "    write_array_to_binary_file(centroids, cluster_center_path)\n",
    "    print(\"cluster center has written to:\",cluster_center_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start create index folder sift_base_1000\n",
      "Create Done\n"
     ]
    }
   ],
   "source": [
    "#生成DiskANN索引\n",
    "#生成索引文件夹\n",
    "folder_name = \"%s_%d\"%(dataset_name,cluster_count)\n",
    "cmd = \"mkdir \"+folder_name\n",
    "print(\"Start create index folder\",folder_name)\n",
    "os_ret = os.system(cmd)\n",
    "time.sleep(0.2)\n",
    "if(os_ret != 0):#如果生成失败\n",
    "    print(\"Create Error! Delete then create\")\n",
    "    os.system(\"rm -r \"+folder_name)\n",
    "    os.system(cmd)\n",
    "    time.sleep(0.2)\n",
    "print(\"Create Done\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cmd: /home/gary/Code/DiskANN/build/apps/build_disk_index --data_type float --dist_fn l2 --data_path /home/gary/Code/vector-ssd/hybrid-index-host/cluster_centers.bin --index_path_prefix sift_base_1000/ -R 32 -L50 -B 120 -M 120\n"
     ]
    }
   ],
   "source": [
    "#构建DiskANN索引构建指令\n",
    "diskann_build_cmd  = diskann_path\n",
    "diskann_build_cmd += \" --data_type\"\n",
    "if(type == np.float32):\n",
    "    diskann_build_cmd += \" float\"\n",
    "elif(type == np.uint8):\n",
    "    diskann_build_cmd += \" uint8\"\n",
    "else:\n",
    "    print(\"Unsupported data type!\",type)\n",
    "diskann_build_cmd += \" --dist_fn l2\" #暂不支持其他类型的距离范式\n",
    "diskann_build_cmd += \" --data_path \" + cluster_center_path\n",
    "diskann_build_cmd += \" --index_path_prefix \" + folder_name + \"/\"\n",
    "diskann_build_cmd += \" -R %d\"%R\n",
    "diskann_build_cmd += \" -L%d\"%L\n",
    "diskann_build_cmd += \" -B %d\"%B\n",
    "diskann_build_cmd += \" -M %d\"%M\n",
    "\n",
    "print(\"cmd:\",diskann_build_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting index build: R=32 L=50 Query RAM budget: 1.28581e+11 Indexing ram budget: 120 T: 48\n",
      "Compressing 128-dimensional data into 128 bytes per vector.\n",
      "Opened: /home/gary/Code/vector-ssd/hybrid-index-host/cluster_centers.bin, size: 512008, cache_size: 512008\n",
      "Training data with 1000 samples loaded.\n",
      "Processing chunk 0 with dimensions [0, 1)\n",
      "Residuals unchanged: 5.8325 becomes 5.8325. Early termination.\n",
      "Processing chunk 1 with dimensions [1, 2)\n",
      "Residuals unchanged: 1.39155 becomes 1.39155. Early termination.\n",
      "Processing chunk 2 with dimensions [2, 3)\n",
      "Residuals unchanged: 0.850481 becomes 0.850481. Early termination.\n",
      "Processing chunk 3 with dimensions [3, 4)\n",
      "Residuals unchanged: 0.803201 becomes 0.803201. Early termination.\n",
      "Processing chunk 4 with dimensions [4, 5)\n",
      "Residuals unchanged: 0.972677 becomes 0.972677. Early termination.\n",
      "Processing chunk 5 with dimensions [5, 6)\n",
      "Residuals unchanged: 1.20779 becomes 1.20779. Early termination.\n",
      "Processing chunk 6 with dimensions [6, 7)\n",
      "Residuals unchanged: 1.87238 becomes 1.87238. Early termination.\n",
      "Processing chunk 7 with dimensions [7, 8)\n",
      "Residuals unchanged: 1.69857 becomes 1.69857. Early termination.\n",
      "Processing chunk 8 with dimensions [8, 9)\n",
      "Residuals unchanged: 7.43759 becomes 7.43759. Early termination.\n",
      "Processing chunk 9 with dimensions [9, 10)\n",
      "Residuals unchanged: 6.47325 becomes 6.47325. Early termination.\n",
      "Processing chunk 10 with dimensions [10, 11)\n",
      "Residuals unchanged: 6.85558 becomes 6.85558. Early termination.\n",
      "Processing chunk 11 with dimensions [11, 12)\n",
      "Residuals unchanged: 3.1064 becomes 3.1064. Early termination.\n",
      "Processing chunk 12 with dimensions [12, 13)\n",
      "Residuals unchanged: 1.94981 becomes 1.94981. Early termination.\n",
      "Processing chunk 13 with dimensions [13, 14)\n",
      "Residuals unchanged: 6.28013 becomes 6.28013. Early termination.\n",
      "Processing chunk 14 with dimensions [14, 15)\n",
      "Residuals unchanged: 7.33102 becomes 7.33102. Early termination.\n",
      "Processing chunk 15 with dimensions [15, 16)\n",
      "Residuals unchanged: 3.34481 becomes 3.34481. Early termination.\n",
      "Processing chunk 16 with dimensions [16, 17)\n",
      "Residuals unchanged: 10.0328 becomes 10.0328. Early termination.\n",
      "Processing chunk 17 with dimensions [17, 18)\n",
      "Residuals unchanged: 3.91351 becomes 3.91351. Early termination.\n",
      "Processing chunk 18 with dimensions [18, 19)\n",
      "Residuals unchanged: 6.67622 becomes 6.67622. Early termination.\n",
      "Processing chunk 19 with dimensions [19, 20)\n",
      "Residuals unchanged: 5.61041 becomes 5.61041. Early termination.\n",
      "Processing chunk 20 with dimensions [20, 21)\n",
      "Residuals unchanged: 2.04286 becomes 2.04286. Early termination.\n",
      "Processing chunk 21 with dimensions [21, 22)\n",
      "Residuals unchanged: 3.30277 becomes 3.30277. Early termination.\n",
      "Processing chunk 22 with dimensions [22, 23)\n",
      "Residuals unchanged: 5.58835 becomes 5.58835. Early termination.\n",
      "Processing chunk 23 with dimensions [23, 24)\n",
      "Residuals unchanged: 6.76729 becomes 6.76729. Early termination.\n",
      "Processing chunk 24 with dimensions [24, 25)\n",
      "Residuals unchanged: 5.85059 becomes 5.85059. Early termination.\n",
      "Processing chunk 25 with dimensions [25, 26)\n",
      "Residuals unchanged: 1.39355 becomes 1.39355. Early termination.\n",
      "Processing chunk 26 with dimensions [26, 27)\n",
      "Residuals unchanged: 2.01278 becomes 2.01278. Early termination.\n",
      "Processing chunk 27 with dimensions [27, 28)\n",
      "Residuals unchanged: 1.14986 becomes 1.14986. Early termination.\n",
      "Processing chunk 28 with dimensions [28, 29)\n",
      "Residuals unchanged: 1.2781 becomes 1.2781. Early termination.\n",
      "Processing chunk 29 with dimensions [29, 30)\n",
      "Residuals unchanged: 0.75546 becomes 0.75546. Early termination.\n",
      "Processing chunk 30 with dimensions [30, 31)\n",
      "Residuals unchanged: 0.604908 becomes 0.604908. Early termination.\n",
      "Processing chunk 31 with dimensions [31, 32)\n",
      "Residuals unchanged: 1.78949 becomes 1.78949. Early termination.\n",
      "Processing chunk 32 with dimensions [32, 33)\n",
      "Residuals unchanged: 12.7154 becomes 12.7154. Early termination.\n",
      "Processing chunk 33 with dimensions [33, 34)\n",
      "Residuals unchanged: 4.22268 becomes 4.22268. Early termination.\n",
      "Processing chunk 34 with dimensions [34, 35)\n",
      "Residuals unchanged: 1.52785 becomes 1.52785. Early termination.\n",
      "Processing chunk 35 with dimensions [35, 36)\n",
      "Residuals unchanged: 2.53614 becomes 2.53614. Early termination.\n",
      "Processing chunk 36 with dimensions [36, 37)\n",
      "Residuals unchanged: 4.26703 becomes 4.26703. Early termination.\n",
      "Processing chunk 37 with dimensions [37, 38)\n",
      "Residuals unchanged: 6.0144 becomes 6.0144. Early termination.\n",
      "Processing chunk 38 with dimensions [38, 39)\n",
      "Residuals unchanged: 4.58893 becomes 4.58893. Early termination.\n",
      "Processing chunk 39 with dimensions [39, 40)\n",
      "Residuals unchanged: 3.37535 becomes 3.37535. Early termination.\n",
      "Processing chunk 40 with dimensions [40, 41)\n",
      "Residuals unchanged: 10.809 becomes 10.809. Early termination.\n",
      "Processing chunk 41 with dimensions [41, 42)\n",
      "Residuals unchanged: 7.56384 becomes 7.56384. Early termination.\n",
      "Processing chunk 42 with dimensions [42, 43)\n",
      "Processing chunk 43 with dimensions [43, 44)\n",
      "Residuals unchanged: 4.03131 becomes 4.03131. Early termination.\n",
      "Processing chunk 44 with dimensions [44, 45)\n",
      "Residuals unchanged: 5.13253 becomes 5.13253. Early termination.\n",
      "Processing chunk 45 with dimensions [45, 46)\n",
      "Residuals unchanged: 6.62904 becomes 6.62904. Early termination.\n",
      "Processing chunk 46 with dimensions [46, 47)\n",
      "Residuals unchanged: 6.88178 becomes 6.88178. Early termination.\n",
      "Processing chunk 47 with dimensions [47, 48)\n",
      "Residuals unchanged: 3.31812 becomes 3.31812. Early termination.\n",
      "Processing chunk 48 with dimensions [48, 49)\n",
      "Residuals unchanged: 10.8689 becomes 10.8689. Early termination.\n",
      "Processing chunk 49 with dimensions [49, 50)\n",
      "Residuals unchanged: 3.4044 becomes 3.4044. Early termination.\n",
      "Processing chunk 50 with dimensions [50, 51)\n",
      "Processing chunk 51 with dimensions [51, 52)\n",
      "Residuals unchanged: 6.81428 becomes 6.81428. Early termination.\n",
      "Processing chunk 52 with dimensions [52, 53)\n",
      "Residuals unchanged: 5.32663 becomes 5.32663. Early termination.\n",
      "Processing chunk 53 with dimensions [53, 54)\n",
      "Residuals unchanged: 3.58223 becomes 3.58223. Early termination.\n",
      "Processing chunk 54 with dimensions [54, 55)\n",
      "Residuals unchanged: 6.25534 becomes 6.25534. Early termination.\n",
      "Processing chunk 55 with dimensions [55, 56)\n",
      "Residuals unchanged: 9.48742 becomes 9.48742. Early termination.\n",
      "Processing chunk 56 with dimensions [56, 57)\n",
      "Residuals unchanged: 11.9365 becomes 11.9365. Early termination.\n",
      "Processing chunk 57 with dimensions [57, 58)\n",
      "Residuals unchanged: 2.37376 becomes 2.37376. Early termination.\n",
      "Processing chunk 58 with dimensions [58, 59)\n",
      "Residuals unchanged: 4.50445 becomes 4.50445. Early termination.\n",
      "Processing chunk 59 with dimensions [59, 60)\n",
      "Residuals unchanged: 6.65583 becomes 6.65583. Early termination.\n",
      "Processing chunk 60 with dimensions [60, 61)\n",
      "Residuals unchanged: 5.04655 becomes 5.04655. Early termination.\n",
      "Processing chunk 61 with dimensions [61, 62)\n",
      "Residuals unchanged: 3.09529 becomes 3.09529. Early termination.\n",
      "Processing chunk 62 with dimensions [62, 63)\n",
      "Residuals unchanged: 1.52451 becomes 1.52451. Early termination.\n",
      "Processing chunk 63 with dimensions [63, 64)\n",
      "Residuals unchanged: 5.40815 becomes 5.40815. Early termination.\n",
      "Processing chunk 64 with dimensions [64, 65)\n",
      "Residuals unchanged: 13.2119 becomes 13.2119. Early termination.\n",
      "Processing chunk 65 with dimensions [65, 66)\n",
      "Residuals unchanged: 2.0874 becomes 2.0874. Early termination.\n",
      "Processing chunk 66 with dimensions [66, 67)\n",
      "Residuals unchanged: 4.25151 becomes 4.25151. Early termination.\n",
      "Processing chunk 67 with dimensions [67, 68)\n",
      "Residuals unchanged: 4.54181 becomes 4.54181. Early termination.\n",
      "Processing chunk 68 with dimensions [68, 69)\n",
      "Residuals unchanged: 4.83421 becomes 4.83421. Early termination.\n",
      "Processing chunk 69 with dimensions [69, 70)\n",
      "Residuals unchanged: 2.43066 becomes 2.43066. Early termination.\n",
      "Processing chunk 70 with dimensions [70, 71)\n",
      "Residuals unchanged: 1.55252 becomes 1.55252. Early termination.\n",
      "Processing chunk 71 with dimensions [71, 72)\n",
      "Residuals unchanged: 5.21365 becomes 5.21365. Early termination.\n",
      "Processing chunk 72 with dimensions [72, 73)\n",
      "Processing chunk 73 with dimensions [73, 74)\n",
      "Residuals unchanged: 2.61889 becomes 2.61889. Early termination.\n",
      "Processing chunk 74 with dimensions [74, 75)\n",
      "Residuals unchanged: 8.28631 becomes 8.28631. Early termination.\n",
      "Processing chunk 75 with dimensions [75, 76)\n",
      "Residuals unchanged: 6.1203 becomes 6.1203. Early termination.\n",
      "Processing chunk 76 with dimensions [76, 77)\n",
      "Residuals unchanged: 5.16587 becomes 5.16587. Early termination.\n",
      "Processing chunk 77 with dimensions [77, 78)\n",
      "Residuals unchanged: 3.53124 becomes 3.53124. Early termination.\n",
      "Processing chunk 78 with dimensions [78, 79)\n",
      "Residuals unchanged: 7.94928 becomes 7.94928. Early termination.\n",
      "Processing chunk 79 with dimensions [79, 80)\n",
      "Residuals unchanged: 8.06045 becomes 8.06045. Early termination.\n",
      "Processing chunk 80 with dimensions [80, 81)\n",
      "Residuals unchanged: 12.7695 becomes 12.7695. Early termination.\n",
      "Processing chunk 81 with dimensions [81, 82)\n",
      "Residuals unchanged: 8.59775 becomes 8.59775. Early termination.\n",
      "Processing chunk 82 with dimensions [82, 83)\n",
      "Residuals unchanged: 8.03812 becomes 8.03812. Early termination.\n",
      "Processing chunk 83 with dimensions [83, 84)\n",
      "Residuals unchanged: 4.49662 becomes 4.49662. Early termination.\n",
      "Processing chunk 84 with dimensions [84, 85)\n",
      "Residuals unchanged: 5.42062 becomes 5.42062. Early termination.\n",
      "Processing chunk 85 with dimensions [85, 86)\n",
      "Residuals unchanged: 5.80112 becomes 5.80112. Early termination.\n",
      "Processing chunk 86 with dimensions [86, 87)\n",
      "Residuals unchanged: 7.81964 becomes 7.81964. Early termination.\n",
      "Processing chunk 87 with dimensions [87, 88)\n",
      "Residuals unchanged: 3.04728 becomes 3.04728. Early termination.\n",
      "Processing chunk 88 with dimensions [88, 89)\n",
      "Residuals unchanged: 13.0366 becomes 13.0366. Early termination.\n",
      "Processing chunk 89 with dimensions [89, 90)\n",
      "Residuals unchanged: 3.71769 becomes 3.71769. Early termination.\n",
      "Processing chunk 90 with dimensions [90, 91)\n",
      "Residuals unchanged: 1.55577 becomes 1.55577. Early termination.\n",
      "Processing chunk 91 with dimensions [91, 92)\n",
      "Residuals unchanged: 2.72237 becomes 2.72237. Early termination.\n",
      "Processing chunk 92 with dimensions [92, 93)\n",
      "Residuals unchanged: 4.66554 becomes 4.66554. Early termination.\n",
      "Processing chunk 93 with dimensions [93, 94)\n",
      "Residuals unchanged: 6.44792 becomes 6.44792. Early termination.\n",
      "Processing chunk 94 with dimensions [94, 95)\n",
      "Residuals unchanged: 4.78406 becomes 4.78406. Early termination.\n",
      "Processing chunk 95 with dimensions [95, 96)\n",
      "Residuals unchanged: 3.88726 becomes 3.88726. Early termination.\n",
      "Processing chunk 96 with dimensions [96, 97)\n",
      "Residuals unchanged: 5.31781 becomes 5.31781. Early termination.\n",
      "Processing chunk 97 with dimensions [97, 98)\n",
      "Residuals unchanged: 1.47456 becomes 1.47456. Early termination.\n",
      "Processing chunk 98 with dimensions [98, 99)\n",
      "Residuals unchanged: 2.03997 becomes 2.03997. Early termination.\n",
      "Processing chunk 99 with dimensions [99, 100)\n",
      "Residuals unchanged: 1.11214 becomes 1.11214. Early termination.\n",
      "Processing chunk 100 with dimensions [100, 101)\n",
      "Residuals unchanged: 0.856363 becomes 0.856363. Early termination.\n",
      "Processing chunk 101 with dimensions [101, 102)\n",
      "Residuals unchanged: 0.613012 becomes 0.613012. Early termination.\n",
      "Processing chunk 102 with dimensions [102, 103)\n",
      "Residuals unchanged: 0.583926 becomes 0.583926. Early termination.\n",
      "Processing chunk 103 with dimensions [103, 104)\n",
      "Residuals unchanged: 1.74685 becomes 1.74685. Early termination.\n",
      "Processing chunk 104 with dimensions [104, 105)\n",
      "Residuals unchanged: 8.45247 becomes 8.45247. Early termination.\n",
      "Processing chunk 105 with dimensions [105, 106)\n",
      "Residuals unchanged: 4.89802 becomes 4.89802. Early termination.\n",
      "Processing chunk 106 with dimensions [106, 107)\n",
      "Processing chunk 107 with dimensions [107, 108)\n",
      "Residuals unchanged: 5.12956 becomes 5.12956. Early termination.\n",
      "Processing chunk 108 with dimensions [108, 109)\n",
      "Residuals unchanged: 1.87102 becomes 1.87102. Early termination.\n",
      "Processing chunk 109 with dimensions [109, 110)\n",
      "Residuals unchanged: 3.14022 becomes 3.14022. Early termination.\n",
      "Processing chunk 110 with dimensions [110, 111)\n",
      "Residuals unchanged: 6.41372 becomes 6.41372. Early termination.\n",
      "Processing chunk 111 with dimensions [111, 112)\n",
      "Residuals unchanged: 8.83746 becomes 8.83746. Early termination.\n",
      "Processing chunk 112 with dimensions [112, 113)\n",
      "Residuals unchanged: 7.02869 becomes 7.02869. Early termination.\n",
      "Processing chunk 113 with dimensions [113, 114)\n",
      "Residuals unchanged: 5.85733 becomes 5.85733. Early termination.\n",
      "Processing chunk 114 with dimensions [114, 115)\n",
      "Residuals unchanged: 5.97208 becomes 5.97208. Early termination.\n",
      "Processing chunk 115 with dimensions [115, 116)\n",
      "Residuals unchanged: 3.54791 becomes 3.54791. Early termination.\n",
      "Processing chunk 116 with dimensions [116, 117)\n",
      "Residuals unchanged: 2.02887 becomes 2.02887. Early termination.\n",
      "Processing chunk 117 with dimensions [117, 118)\n",
      "Residuals unchanged: 6.4044 becomes 6.4044. Early termination.\n",
      "Processing chunk 118 with dimensions [118, 119)\n",
      "Residuals unchanged: 7.33449 becomes 7.33449. Early termination.\n",
      "Processing chunk 119 with dimensions [119, 120)\n",
      "Residuals unchanged: 3.36121 becomes 3.36121. Early termination.\n",
      "Processing chunk 120 with dimensions [120, 121)\n",
      "Residuals unchanged: 6.23963 becomes 6.23963. Early termination.\n",
      "Processing chunk 121 with dimensions [121, 122)\n",
      "Residuals unchanged: 1.14682 becomes 1.14682. Early termination.\n",
      "Processing chunk 122 with dimensions [122, 123)\n",
      "Residuals unchanged: 1.03037 becomes 1.03037. Early termination.\n",
      "Processing chunk 123 with dimensions [123, 124)\n",
      "Residuals unchanged: 1.00402 becomes 1.00402. Early termination.\n",
      "Processing chunk 124 with dimensions [124, 125)\n",
      "Residuals unchanged: 1.15402 becomes 1.15402. Early termination.\n",
      "Processing chunk 125 with dimensions [125, 126)\n",
      "Residuals unchanged: 1.22108 becomes 1.22108. Early termination.\n",
      "Processing chunk 126 with dimensions [126, 127)\n",
      "Residuals unchanged: 1.73099 becomes 1.73099. Early termination.\n",
      "Processing chunk 127 with dimensions [127, 128)\n",
      "Residuals unchanged: 1.96761 becomes 1.96761. Early termination.\n",
      "Writing bin: sift_base_1000/_pq_pivots.bin\n",
      "bin: #pts = 256, #dims = 128, size = 131080B\n",
      "Finished writing bin.\n",
      "Writing bin: sift_base_1000/_pq_pivots.bin\n",
      "bin: #pts = 128, #dims = 1, size = 520B\n",
      "Finished writing bin.\n",
      "Writing bin: sift_base_1000/_pq_pivots.bin\n",
      "bin: #pts = 129, #dims = 1, size = 524B\n",
      "Finished writing bin.\n",
      "Writing bin: sift_base_1000/_pq_pivots.bin\n",
      "bin: #pts = 4, #dims = 1, size = 40B\n",
      "Finished writing bin.\n",
      "Saved pq pivot data to sift_base_1000/_pq_pivots.bin of size 136220B.\n",
      "Opened: /home/gary/Code/vector-ssd/hybrid-index-host/cluster_centers.bin, size: 512008, cache_size: 512008\n",
      "Reading bin file sift_base_1000/_pq_pivots.bin ...\n",
      "Opening bin file sift_base_1000/_pq_pivots.bin... \n",
      "Metadata: #pts = 4, #dims = 1...\n",
      "done.\n",
      "Reading bin file sift_base_1000/_pq_pivots.bin ...\n",
      "Opening bin file sift_base_1000/_pq_pivots.bin... \n",
      "Metadata: #pts = 256, #dims = 128...\n",
      "done.\n",
      "Reading bin file sift_base_1000/_pq_pivots.bin ...\n",
      "Opening bin file sift_base_1000/_pq_pivots.bin... \n",
      "Metadata: #pts = 128, #dims = 1...\n",
      "done.\n",
      "Reading bin file sift_base_1000/_pq_pivots.bin ...\n",
      "Opening bin file sift_base_1000/_pq_pivots.bin... \n",
      "Metadata: #pts = 129, #dims = 1...\n",
      "done.\n",
      "Loaded PQ pivot information\n",
      "Processing points  [0, 1000)...done.\n",
      "Time for generating quantized data: 2.806268 seconds\n",
      "Full index fits in RAM budget, should consume at most 0.000744164GiBs, so building in one shot\n",
      "L2: Using AVX2 distance computation DistanceL2Float\n",
      "Passed, empty search_params while creating index config\n",
      "Using only first 1000 from file.. \n",
      "Starting index build with 1000 points... \n",
      "0% of index build completed.Starting final cleanup..done. Link time: 0.110854s\n",
      "Index built with degree: max:32  avg:32  min:32  count(deg<2):0\n",
      "Not saving tags as they are not enabled.\n",
      "Time taken for save: 0.008122s.\n",
      "Time for building merged vamana index: 0.120279 seconds\n",
      "Opened: /home/gary/Code/vector-ssd/hybrid-index-host/cluster_centers.bin, size: 512008, cache_size: 512008\n",
      "Vamana index file size=132024\n",
      "Opened: sift_base_1000/_disk.index, cache_size: 67108864\n",
      "medoid: 643B\n",
      "max_node_len: 644B\n",
      "nnodes_per_sector: 6B\n",
      "# sectors: 167\n",
      "Sector #0written\n",
      "Finished writing 688128B\n",
      "Writing bin: sift_base_1000/_disk.index\n",
      "bin: #pts = 9, #dims = 1, size = 80B\n",
      "Finished writing bin.\n",
      "Output disk index file written to sift_base_1000/_disk.index\n",
      "Finished writing 688128B\n",
      "Time for generating disk layout: 0.039171 seconds\n",
      "Opened: /home/gary/Code/vector-ssd/hybrid-index-host/cluster_centers.bin, size: 512008, cache_size: 512008\n",
      "Loading base /home/gary/Code/vector-ssd/hybrid-index-host/cluster_centers.bin. #points: 1000. #dim: 128.\n",
      "Wrote 107 points to sample file: sift_base_1000/_sample_data.bin\n",
      "Indexing time: 2.9677\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#执行建图指令\n",
    "os.system(diskann_build_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取出DiskANN建图的结果\n",
    "graph_path = folder_name + \"/_disk.index\"\n",
    "pq_compressed_path = folder_name + \"/_pq_compressed.bin\"\n",
    "pq_pivots_path = folder_name + \"/_pq_pivots.bin\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sift_base_1000/_pq_pivots.bin'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
